package com.oneday.ashop.core.base.init;

import com.oneday.ashop.core.base.mongo.BaseMongoDaoSupport;
import com.oneday.ashop.core.entity.book.Book;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.DateUtils;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/**
 * 初始化book,实际上这算是 一个脚本用来抓取douban的图书信息
 * User:bob.yang
 * Date 2015/8/25 0025
 */
@Component
@Slf4j
public class BookDataInit extends BaseMongoDaoSupport{

    String author = "作者:";
    String publisht;
    public  List<String> list = new ArrayList<String>(){{add("作者:"); add("出版社:");
    add("副标题:"); add("原作名:"); add("译者:"); add("丛书:"); add("定价:"); add("出版年:"); add("页数:"); add("装帧:"); add("装帧:");
    add("ISBN:");}};

    int requestError = 0;
    int solverError = 0;
    int success = 0;

    //失败的地址
    public List<String> failedUrls = new ArrayList<>();

   //@PostConstruct
    public void init() {
        //获得一个默认的HttpClient客户端
        RequestConfig defaultRequestConfig = RequestConfig.custom()
                .setSocketTimeout(5000)
                .setConnectTimeout(5000)
                .setConnectionRequestTimeout(5000)
                .setCookieSpec(CookieSpecs.STANDARD_STRICT)
                .build();
        CloseableHttpClient httpClient = HttpClients.custom()
                .setDefaultRequestConfig(defaultRequestConfig)
                .build();

        //读取所有需要爬取的地址
        File file = new File("D://goUrl.text");
        FileReader fileReader = null;
        try {
            fileReader = new FileReader(file);
        } catch (FileNotFoundException e) {
           log.error("没有这个文件！",e);
            return;
        }
        BufferedReader reader = new BufferedReader(fileReader);
        String str = null;
        List<String> urls = new ArrayList<>();
        try {
            while((str = reader.readLine()) != null) {
                urls.add(str);
            }
        } catch (IOException e) {
           log.error("读取文件异常！",e);
            return;
        }
        for(String url : urls) {

            //模拟浏览器
            HttpGet httpGet =   new HttpGet(url);
            httpGet.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            httpGet.setHeader("Accept-Encoding","gzip,deflate,sdch");
            httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.8");
            httpGet.setHeader("Connection","keep-alive");
            httpGet.setHeader("Cache-Control","max-age=0");
//                httpGet.setHeader("Host","movie.douban.com");
            httpGet.setHeader("Cookie", "bid=\"vsXMg4Bay30\"; ll=\"108296\"; _ga=GA1.2.1038398848.1430800075; ps=y; viewed=\"26569724_26390842_26451423_5975576_26392727_25773187_26413027_4010969_24722612_26274206\"; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1440396583%2C%22http%3A%2F%2Faccounts.douban.com%2Flogin%3Fuid%3D51295828%26alias%3D1245964132%2540qq.com%26redir%3Dhttp%253A%252F%252Fwww.douban.com%252F%26source%3Dindex_nav%26error%3D1013%22%5D; ue=\"1245964132@qq.com\"; dbcl2=\"51295828:5qKM2LBoMeY\"; ck=\"YWLZ\"; ap=1; push_noty_num=0; push_doumail_num=0; _pk_id.100001.8cb4=84049da8a64e0f42.1438736478.36.1440407338.1440392367.; _pk_ses.100001.8cb4=*; __utma=30149280.1038398848.1430800075.1440391513.1440396243.37; __utmb=30149280.68.10.1440396243; __utmc=30149280; __utmz=30149280.1439520983.20.8.utmcsr=douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utmv=30149280.5129");
            httpGet.setHeader("Upgrade-Insecure-Requests", "1");
            httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36");

            //发送请求
            CloseableHttpResponse response = null;
            try {
                response = httpClient.execute(httpGet);
//                Thread.sleep(1000);
            } catch (Exception e) {
                requestError ++;
                failedUrls.add(url);
                log.error("--------HttpClient请求:{}异常！", url, e);
                log.info("-------进入下一个页面....");
                try {
                    response.close();
                    log.info("关闭连接！");
                } catch (Exception e1) {
                    continue;
                }
                continue;
            }
            //判断返回的状态
            if(response.getStatusLine().getStatusCode() != 200) {
               log.info("请求页面{}返回：{}", url, response.getStatusLine());
                try {
                    response.close();
                } catch (IOException e) {
                    continue;
                }
                continue;
            }
            try{
                getResource(response.getEntity());
                response.close();
            }catch (Exception e) {
                try {
                    response.close();
                } catch (Exception e1) {
                }
                solverError ++;
                failedUrls.add(url);
                log.error("--------解析页面{}异常", url, e);
                log.info("-------进入下一个页面....");
                continue;
            }
        }
        System.out.println("请求失败：" + requestError);
        System.out.println("解析失败：" + solverError);
        System.out.println("成功插入：" + success);
        File errorFiel = new File("D://failedUrl.text");
        FileWriter fileWriter = null;
        try {
            fileWriter = new FileWriter(errorFiel);
        } catch (IOException e) {
            e.printStackTrace();
        }
        BufferedWriter bw = new BufferedWriter(fileWriter);
        for(String url : failedUrls) {
            try {
                bw.write(url);
                bw.newLine();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        try {
            bw.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }


    //抓取需要的资源
    public  void getResource(HttpEntity entity) throws IOException {
        //生成书的实体
        Book book = new Book();

        String webContent = EntityUtils.toString(entity);
        entity.getContent().close();
        //首先抓取需要的图片
        Document doc = Jsoup.parse(webContent);
        Elements elements = doc.select("div#mainpic"); //先根据id选择相应的元素
        doc = Jsoup.parse(elements.toString());
        Elements elements1 = doc.select("a[href]"); //再在相应的元素内选择想要的元素
        for(Element node : elements1){
            String href = node.attr("href"); //把元素的内容取出来
            book.setImage(href); //封面
        }

        //抓取书名
        doc = Jsoup.parse(webContent);
        elements = doc.select("span[property=v:itemreviewed]");
        for(Element node : elements) {
            String content = node.text(); //获得该元素下的文本信息
            book.setName(content);
        }

        //然后再抓取图书的介绍
        doc = Jsoup.parse(webContent);
        elements = doc.select("div#info");
        for(Element node : elements) {
            String content = node.text(); //获得该元素下的文本信息
            String[] array = content.split(" ");
            int i = 0;
            String subject = null;
            String authorTemp = null;
            for(String str : array) {
                if(list.contains(str)) {
                    subject = str;
                }else {
                   if(subject.equals("作者:")) {
                       //因为有的有地区，有的没有 这里做特殊处理
                       if(str.contains("]")) {
                           book.setArea(authorTemp);
                       }else {
                           if(book.getAuthor() == null) {
                               book.setAuthor(str);
                           }else {
                               book.setAuthor(book.getAuthor()+str);
                           }
                       }

                   }
                    if(subject.equals("出版社:")) {
                        book.setPublisher(str);
                    }
                    if(subject.equals("副标题:")) {
                        if(book.getSubName() == null) {
                            book.setSubName(str);
                        }else {
                            book.setSubName(book.getSubName() + str);
                        }
                    }
                    if(subject.equals("原作名:")) {
                        if(book.getOriName() == null) {
                            book.setOriName(str);
                        }else {
                            book.setOriName(book.getOriName() + str);
                        }
                    }
                    if(subject.equals("译者:")) {
                        if(book.getTranslator() == null) {
                            book.setTranslator(str);
                        }else {
                            book.setTranslator(book.getTranslator() + str);
                        }
                    }
                    if(subject.equals("出版年:")) {
                        book.setPublishDate(str);
                    }
                    if(subject.equals("页数:")) {
                        book.setPageNum(str);
                    }
                    if(subject.equals("定价:")) {
                        book.setPrice(str);
                    }
                    if(subject.equals("装帧:")) {
                        if(book.getBindingLayout() == null) {
                            book.setBindingLayout(str);
                        }
                    }
                    if(subject.equals("丛书:")) {
                        if(book.getSeries() == null) {
                            book.setSeries(str);
                        }else {
                            book.setSeries(book.getSeries() + str);
                        }
                    }
                    if(subject.equals("ISBN:")) {
                        book.setISBN(str);
                    }
                }

//                if(i == 1) {
//                    book.setArea(str); // 地区
//                }
//                if(i == 2 ) {
//                    book.setAuthor(str); //作者
//                }
//                if(i == 4) {
//                    book.setPublisher(str); //出版社
//                }
//                if(i == 6) {
//                    book.setTranslator(str); //译者
//                }
//                if(i == 8) {
//                    book.setPublishDate(str); //出版年息
//                }
//                if(i == 10) {
//                    book.setPageNum(Long.parseLong(str)); //页数
//                }
//                if(i == 12) {
//                    book.setPrice(str); //定价
//                }
//                if(i == 14) {
//                    book.setBindingLayout(str); //装帧
//                }
//                if(i == 15) {
//                    book.setSeries(str); //系列
//                }
//                if(i == 17) {
//                    book.setISBN(str); //ISBN
//                }
//                i++;
            }
        }
        //抓入图书的简介
        doc = Jsoup.parse(webContent);
        elements = doc.select("div#link-report");
        doc = Jsoup.parse(elements.toString());
        elements = doc.select("span[class=all hidden]"); //根据元素的属性值拉取内容
        for(Element node : elements) {
            String content = node.text(); //获得该元素下的文本信息
            String[] array = content.split(" ");
            StringBuffer sb = new StringBuffer();
            int i = 0;
            for(String str : array) {
                if(i != 0) {
                    sb.append("/r/n"); //换行
                    sb.append(str);
                }else {
                    sb.append(str);
                }
                i ++;
            }
            book.setIntroduction(sb.toString());
        }
        if(book.getName() == null || "".equals(book.getName())) {
            return;
        }
        //插入数据库
        insert(book);
        success ++;
    }

    //过滤掉不合法的地址
    public static void checkUrl(String url) {

    }

}
